{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基于SVD的协同过滤"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 导入工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#load数据（用户和物品索引，以及倒排表）\n",
    "import pickle\n",
    "import json  \n",
    "\n",
    "from numpy.random import random\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读入数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#用户和item的索引\n",
    "users_index = pickle.load(open(\"user_index.pkl\", 'rb'))\n",
    "items_index = pickle.load(open(\"items_index.pkl\", 'rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "    \n",
    "#用户-物品关系矩阵R\n",
    "#scores = sio.mmread(\"scores\").todense()\n",
    "    \n",
    "#倒排表\n",
    "##每个用户打过分的电影\n",
    "user_items = pickle.load(open(\"user_items.pkl\", 'rb'))\n",
    "##对每个电影打过分的事用户\n",
    "item_users = pickle.load(open(\"item_users.pkl\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOCKSGZ12A58A7CA4B</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOCVTLJ12A6310F0FD</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SODLLYS12A8C13A96B</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOEGIYH12A6D4FC0E3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4e11f45d732f4861772b2906f81a7d384552ad12</td>\n",
       "      <td>SOFRQTD12A81C233C0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  play_count\n",
       "0  4e11f45d732f4861772b2906f81a7d384552ad12  SOCKSGZ12A58A7CA4B           1\n",
       "1  4e11f45d732f4861772b2906f81a7d384552ad12  SOCVTLJ12A6310F0FD           1\n",
       "2  4e11f45d732f4861772b2906f81a7d384552ad12  SODLLYS12A8C13A96B           3\n",
       "3  4e11f45d732f4861772b2906f81a7d384552ad12  SOEGIYH12A6D4FC0E3           1\n",
       "4  4e11f45d732f4861772b2906f81a7d384552ad12  SOFRQTD12A81C233C0           2"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取训练数据\n",
    "dpath = './data/'\n",
    "df_triplet = pd.read_csv(dpath +'train.csv')\n",
    "df_triplet.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 初始化模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#隐含变量的维数\n",
    "K = 40\n",
    "\n",
    "#item和用户的偏置项\n",
    "bi = np.zeros((n_items,1))    \n",
    "bu = np.zeros((n_users,1))   \n",
    "\n",
    "#item和用户的隐含向量\n",
    "qi =  np.zeros((n_items,K))    \n",
    "pu =  np.zeros((n_users,K))   \n",
    "\n",
    "\n",
    "for uid in range(n_users):  #对每个用户\n",
    "    pu[uid] = np.reshape(random((K,1))/10*(np.sqrt(K)),K)\n",
    "       \n",
    "for iid in range(n_items):  #对每个item\n",
    "    qi[iid] = np.reshape(random((K,1))/10*(np.sqrt(K)),K)\n",
    "\n",
    "#所有用户的平均打分\n",
    "mu = df_triplet['play_count'].mean()  #average rating"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根据当前参数，预测用户uid对Item（i_id）的打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svd_pred(uid, iid):  \n",
    "    score = mu + bi[iid] + bu[uid] + np.sum(qi[iid]* pu[uid])  \n",
    "        \n",
    "    #将打分范围控制在1-5之间\n",
    "    #if score>5:  \n",
    "        #score = 5  \n",
    "    #elif score<1:  \n",
    "        #score = 1  \n",
    "        \n",
    "    return score  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 0-th  step is running\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/sw/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:38: RuntimeWarning: overflow encountered in multiply\n",
      "/var/sw/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:30: RuntimeWarning: overflow encountered in square\n",
      "/var/sw/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:37: RuntimeWarning: overflow encountered in multiply\n",
      "/var/sw/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: RuntimeWarning: overflow encountered in multiply\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the rmse of this step on train data is  [ nan]\n",
      "The 1-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 2-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 3-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 4-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 5-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 6-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 7-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 8-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 9-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 10-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 11-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 12-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 13-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 14-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 15-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 16-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 17-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 18-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 19-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 20-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 21-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 22-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 23-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 24-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 25-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 26-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 27-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 28-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 29-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 30-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 31-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 32-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 33-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 34-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 35-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 36-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 37-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 38-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 39-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 40-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 41-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 42-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 43-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 44-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 45-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 46-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 47-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 48-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n",
      "The 49-th  step is running\n",
      "the rmse of this step on train data is  [ nan]\n"
     ]
    }
   ],
   "source": [
    "#gamma：为学习率\n",
    "#Lambda：正则参数\n",
    "#steps：迭代次数\n",
    "\n",
    "steps=50\n",
    "gamma=0.04\n",
    "Lambda=0.15\n",
    "\n",
    "#总的打分记录数目\n",
    "n_records = df_triplet.shape[0]\n",
    "\n",
    "for step in range(steps):  \n",
    "    print ('The ' + str(step) + '-th  step is running' )\n",
    "    rmse_sum=0.0 \n",
    "            \n",
    "    #将训练样本打散顺序\n",
    "    kk = np.random.permutation(n_records)  \n",
    "    for j in range(n_records):  \n",
    "        #每次一个训练样本\n",
    "        line = kk[j]  \n",
    "        \n",
    "        uid = users_index [df_triplet.iloc[line]['user']]\n",
    "        iid = items_index [df_triplet.iloc[line]['song']]\n",
    "    \n",
    "        rating  = df_triplet.iloc[line]['play_count']\n",
    "                \n",
    "        #预测残差\n",
    "        eui = rating - svd_pred(uid, iid)  \n",
    "        #残差平方和\n",
    "        rmse_sum += eui**2  \n",
    "                \n",
    "        #随机梯度下降，更新\n",
    "        bu[uid] += gamma * (eui - Lambda * bu[uid])  \n",
    "        bi[iid] += gamma * (eui - Lambda * bi[iid]) \n",
    "                \n",
    "        temp = qi[iid]  \n",
    "        qi[iid] += gamma * (eui* pu[uid]- Lambda*qi[iid] )  \n",
    "        pu[uid] += gamma * (eui* temp - Lambda*pu[uid])  \n",
    "            \n",
    "    #学习率递减\n",
    "    gamma=gamma*0.93  \n",
    "    print (\"the rmse of this step on train data is \",np.sqrt(rmse_sum/n_records))  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 保存模型参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A method for saving object data to JSON file\n",
    "def save_json(filepath):\n",
    "    dict_ = {}\n",
    "    dict_['mu'] = mu\n",
    "    dict_['K'] = K\n",
    "    \n",
    "    dict_['bi'] = bi.tolist()\n",
    "    dict_['bu'] = bu.tolist()\n",
    "    \n",
    "    dict_['qi'] = qi.tolist()\n",
    "    dict_['pu'] = pu.tolist()\n",
    "\n",
    "    # Creat json and save to file\n",
    "    json_txt = json.dumps(dict_)\n",
    "    with open(filepath, 'w') as file:\n",
    "        file.write(json_txt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# A method for loading data from JSON file\n",
    "def load_json(filepath):\n",
    "    with open(filepath, 'r') as file:\n",
    "        dict_ = json.load(file)\n",
    "\n",
    "        mu = dict_['mu']\n",
    "        K = dict_['K']\n",
    "\n",
    "        bi = np.asarray(dict_['bi'])\n",
    "        bu = np.asarray(dict_['bu'])\n",
    "    \n",
    "        qi = np.asarray(dict_['qi'])\n",
    "        pu = np.asarray(dict_['pu'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_json('svd_model.json')\n",
    "load_json('svd_model.json')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对给定用户，推荐物品/计算打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#user：用户\n",
    "#返回推荐items及其打分（DataFrame）\n",
    "def svd_CF_recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    #训练集中该用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "\n",
    "    #该用户对所有item的打分\n",
    "    user_items_scores = np.zeros(n_items)\n",
    "\n",
    "    #预测打分\n",
    "    for i in range(n_items):  # all items \n",
    "        if i not in cur_user_items: #训练集中没打过分\n",
    "            user_items_scores[i] = svd_pred(cur_user_id, i)  #预测打分\n",
    "    \n",
    "    #推荐\n",
    "    #Sort the indices of user_item_scores based upon their value，Also maintain the corresponding score\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_items_scores))), reverse=True)\n",
    "    \n",
    "    #Create a dataframe from the following\n",
    "    columns = ['song', 'score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "         \n",
    "    #Fill the dataframe with top 20 (n_rec_items) item based recommendations\n",
    "    #sort_index = sort_index[0:n_rec_items]\n",
    "    #Fill the dataframe with all items based recommendations\n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1] \n",
    "        cur_item = list (items_index.keys()) [list (items_index.values()).index (cur_item_index)]\n",
    "            \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)]=[cur_item, sort_index[i][0]]\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 读取测试数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>484734ce6667a6a25df23a961d5c5a9458afbfa1</td>\n",
       "      <td>SOWDLPO12A6D4F72BB</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>484734ce6667a6a25df23a961d5c5a9458afbfa1</td>\n",
       "      <td>SOWIMTL12A8C1386DC</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>484734ce6667a6a25df23a961d5c5a9458afbfa1</td>\n",
       "      <td>SOWKRSR12A8C13CA37</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>484734ce6667a6a25df23a961d5c5a9458afbfa1</td>\n",
       "      <td>SOXFYTY127E9433E7D</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>484734ce6667a6a25df23a961d5c5a9458afbfa1</td>\n",
       "      <td>SOXKOIY12A8C13C1EA</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  play_count\n",
       "0  484734ce6667a6a25df23a961d5c5a9458afbfa1  SOWDLPO12A6D4F72BB          30\n",
       "1  484734ce6667a6a25df23a961d5c5a9458afbfa1  SOWIMTL12A8C1386DC           3\n",
       "2  484734ce6667a6a25df23a961d5c5a9458afbfa1  SOWKRSR12A8C13CA37          19\n",
       "3  484734ce6667a6a25df23a961d5c5a9458afbfa1  SOXFYTY127E9433E7D           1\n",
       "4  484734ce6667a6a25df23a961d5c5a9458afbfa1  SOXKOIY12A8C13C1EA           9"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取测试数据\n",
    "\n",
    "dpath = './data/'\n",
    "df_triplet_test = pd.read_csv(dpath +'test.csv')\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 测试，并计算评价指标\n",
    "PR、覆盖度、RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "92d2e8ff105b7b7ddc163e740921cafdbcd815bf is a new user.\n",
      "\n",
      "15b447e75c02e91a9d8ce92fd6f2d1366d07d380 is a new user.\n",
      "\n",
      "6b9ef666e3b10fd428530e519e1a3f68c2fbbd1a is a new user.\n",
      "\n",
      "9784566a61c79b18b5d72584f3cc95ea134d14de is a new user.\n",
      "\n",
      "45d34f6b12ce922136ea7c6a82de3524be24d683 is a new user.\n",
      "\n",
      "7e80a4beb851feae1f2c16a2779f0e9ff424647a is a new user.\n",
      "\n",
      "5588ce55d35d7c70f06e1928ec0967117c5d712b is a new user.\n",
      "\n",
      "6a6d346cc92ee699a765a06fd97db74b864a8129 is a new user.\n",
      "\n",
      "bd3478df2f64daf2e38b0550c988157b2c118481 is a new user.\n",
      "\n",
      "4748dd6561d01b44a5add4c9e1e4282f1578bdcc is a new user.\n",
      "\n",
      "7560076aa3ff4c9a46d917262a87a3d830543469 is a new user.\n",
      "\n",
      "b7c24f770be6b802805ac0e2106624a517643c17 is a new user.\n",
      "\n",
      "92227472de1d2ed14725adff31cfa56a3f29047b is a new user.\n",
      "\n",
      "4f9827b79319210f9e35259bce0587cf04f74a03 is a new user.\n",
      "\n",
      "168db3745c3e7255a1ef0e759fa4e413e5ba9dfc is a new user.\n",
      "\n",
      "8b64bf1db75f427e41133d39955fb3f1a28417d9 is a new user.\n",
      "\n",
      "1fc2a7f42424249718cc544a0a1036a69d5bc7b8 is a new user.\n",
      "\n",
      "7323c1c0cc06f5f77f61db323d00a0a628092848 is a new user.\n",
      "\n",
      "6babcd5c5d0d794480580eab45820e489c344a0f is a new user.\n",
      "\n",
      "87b9680d2db44f901bf79d9a61fb298dbb2a6603 is a new user.\n",
      "\n",
      "e851e806941435c6d5748bd64c1c55e5c0551ab1 is a new user.\n",
      "\n",
      "e6b18cffa2afad9245b7d9eb08390efeebef1f3b is a new user.\n",
      "\n",
      "1991ec49e285545363025ffcb4988c6dd860c766 is a new user.\n",
      "\n",
      "480f3a551bbcdb53f8eb28e44614aa7ce0cc8d0e is a new user.\n",
      "\n",
      "44bf2bb66daf59a20d7b7df64fda29ec16c74cb9 is a new user.\n",
      "\n",
      "59565a9a2a86c12c54bd887c723a47c7f8ab8090 is a new user.\n",
      "\n",
      "6c8faa430c61308b863c22f322ea9fef161f1be8 is a new user.\n",
      "\n",
      "dbb235f944fe78ef8198950e64e4bbc2bdbef172 is a new user.\n",
      "\n",
      "83b94f1d1bf5581499cc0738807c8c41f3bf6706 is a new user.\n",
      "\n",
      "52a6c7b6221f57c89dacbbd06854ca0dc415e9e6 is a new user.\n",
      "\n",
      "e46932327dfffa6ec69dbadbe9dd883a3a383673 is a new user.\n",
      "\n",
      "93743199681d206189b5288a3cb421970e4cd872 is a new user.\n",
      "\n",
      "c405c586f6d7aadbbadfcba5393b543fd99372ff is a new user.\n",
      "\n",
      "33db1f9a79d6400c01112f2b4897eeee533b1b8b is a new user.\n",
      "\n",
      "9254a3fdc569428c3b1c3904db36d485c47e2544 is a new user.\n",
      "\n",
      "e8184db22c453007c34ba8ab7e2a4cda8ca7969b is a new user.\n",
      "\n",
      "421a43936c80f2232747bdcf340e2744fdf37aa6 is a new user.\n",
      "\n",
      "fe67eae6791418a5a85125145609f518f01efe48 is a new user.\n",
      "\n",
      "145f15885973e477a91d2fc560a63850622e0d9e is a new user.\n",
      "\n",
      "90d2fcb1dbe47dc1e9442587e259811a0437a13f is a new user.\n",
      "\n",
      "02471b044e6bdff7c01e1ea2791214268ba5aaf4 is a new user.\n",
      "\n",
      "4626aa787c5e156440970cc12f9ecad4cde3f06b is a new user.\n",
      "\n",
      "0d54fad06b250c41865f6af5b8d35dd5c5750c75 is a new user.\n",
      "\n",
      "a403e8052c47447959193e549dc6b609e6724466 is a new user.\n",
      "\n",
      "627fae8384469c99391b504f9d98030999019e85 is a new user.\n",
      "\n",
      "c2dbf6f4c32dc36af4d6fadb16cd8b79c64a6aa8 is a new user.\n",
      "\n",
      "a18aa09c5b8a1c03d03cdf6d8eb11c2bf5b907cd is a new user.\n",
      "\n",
      "16a169fe2aae8b9def552092527db97a412bee7d is a new user.\n",
      "\n",
      "80f3cceeac5a7266b6d3696c04f21925e72bf9f4 is a new user.\n",
      "\n",
      "41d990857d4659738215aadddb0d6e630685d278 is a new user.\n",
      "\n",
      "aff2b00aeba4a389d22c474dc33645e0a6dfd56e is a new user.\n",
      "\n",
      "18d857219b6fedae682aa3a5e2be18ec526ab241 is a new user.\n",
      "\n",
      "eb75703cc9f9a33c4b1d2f7b3abf9a6be4c732aa is a new user.\n",
      "\n",
      "6b36f65d2eb5579a8b9ed5b4731a7e13b8760722 is a new user.\n",
      "\n",
      "839223f11c98e0c8017e8ecd6fc7b8706658c966 is a new user.\n",
      "\n",
      "22e08d5e101ab5b86dc394856d508e175a5242a6 is a new user.\n",
      "\n",
      "974a05be3d2385f40af4e6e610be59544657085e is a new user.\n",
      "\n",
      "bf82fdc9210bdaa712a5e310a35fe9784d5700e4 is a new user.\n",
      "\n",
      "83522e9d56c3cda41d1853465f1b05f4c2d07550 is a new user.\n",
      "\n",
      "5b68b39d7c8d66935839aa58121abd46605b34cc is a new user.\n",
      "\n",
      "5fb2630c42e4a08f4f5456306d547a1de3c60100 is a new user.\n",
      "\n",
      "68bfa6f926e2b47eefe4ca41a2b1eb7b4d4fc001 is a new user.\n",
      "\n",
      "c7417a59a6d67ef869bf970671b5246c4e3e16d6 is a new user.\n",
      "\n",
      "6eb4c037657215db0d6f4d37bec84fb491374df9 is a new user.\n",
      "\n",
      "074a2197ff72db9f7e44606dfd33208dcdf29f06 is a new user.\n",
      "\n",
      "66abc1ae25dca07b75109164f9dacfe33f9572ba is a new user.\n",
      "\n",
      "3ab78e39bddeaeb789edad041fff03050077417c is a new user.\n",
      "\n",
      "13a2d690b099bcd3fdd256bd39aa14edadd5db08 is a new user.\n",
      "\n",
      "fb644c3f2a83114325dc67b97df0bce60b5ac9a1 is a new user.\n",
      "\n",
      "33e3fc88b0ab07f872bf7515413887374dd76ad6 is a new user.\n",
      "\n",
      "a4a068221b97aad518af038628daf1180e30113a is a new user.\n",
      "\n",
      "37e0e4c4cb5b1e6022908bd129f70c70e08ac68b is a new user.\n",
      "\n",
      "6a3657565b45f103af50ec87321e36712f098aba is a new user.\n",
      "\n",
      "acedfad21e1fd702aacda8111551f00b1f6f378e is a new user.\n",
      "\n",
      "956dc1095d8f22575d3936191ce20b789b0ffc4d is a new user.\n",
      "\n",
      "716ed1ec67d67bfa05db3ffeb641d13f46dca6ec is a new user.\n",
      "\n",
      "2401e90ee6a2796f74cf01c5e808ae8a0540ad61 is a new user.\n",
      "\n",
      "22f6aae94643c2cea285413068f80274e7f1f75e is a new user.\n",
      "\n",
      "7893479225de4c7d4646e90c9bf15f9f624e2cff is a new user.\n",
      "\n",
      "18765abd13462c176d9ccc89e71bfc23265dfed7 is a new user.\n",
      "\n",
      "36bee226881241a38e3c9997cf0c84e2959035e7 is a new user.\n",
      "\n",
      "7be76db45f21dcbb68ff9cc55ebeddcb1db71479 is a new user.\n",
      "\n",
      "148163a8a9c6173a11fb36c3215c1a19251e63c8 is a new user.\n",
      "\n",
      "4d744903c553122fb4815d06e047aba9f974b938 is a new user.\n",
      "\n",
      "752d48df8a89c534e9957d85209837c6a943a14f is a new user.\n",
      "\n",
      "1280b7963657a12b28a8ca58bf736ddeb256fda1 is a new user.\n",
      "\n",
      "49c8369971513c689d0ca6d0382e5c5a710df7e7 is a new user.\n",
      "\n",
      "aec9f74039a6d861551c5e4e0a799a2ec4196c81 is a new user.\n",
      "\n",
      "a05e548059abb1f77cad6cb9c3c0c48e0616f551 is a new user.\n",
      "\n",
      "e7775d30ad499fa204257d1246a7f87a6bfcc80f is a new user.\n",
      "\n",
      "e82b3380f770c78f8f067f464941057c798eaca2 is a new user.\n",
      "\n",
      "dd3696048c81b1aff836a606cda7e14be5bb92a6 is a new user.\n",
      "\n",
      "4cb4632e48cd8960dc113eae340adc402a0413cf is a new user.\n",
      "\n",
      "a4ccc36714975978b545e35db83584fa9f7fa6c6 is a new user.\n",
      "\n",
      "998e2858f6ffb758e298ad2223da4b7a65e676e3 is a new user.\n",
      "\n",
      "f9edc8907be695518817082a224aa43beca7d994 is a new user.\n",
      "\n",
      "bd82eefb8f1ddc894b48ee2baca28c89ff70710d is a new user.\n",
      "\n",
      "3907d4fe152c83b3aeaa28b294f23daf072eca46 is a new user.\n",
      "\n",
      "3996245591e28d4ab3fc572cae1c44c456e2fa34 is a new user.\n",
      "\n",
      "f9cf7849592621b46a793e0f283de8ab48b3d5f8 is a new user.\n",
      "\n",
      "99dedd05d478e3aa54b199f4432fcb9907456b34 is a new user.\n",
      "\n",
      "754f985cb55753e84f1933f3280b06e2bb379847 is a new user.\n",
      "\n",
      "6d147ffb9f1b7d7188cdb964dd15093227818421 is a new user.\n",
      "\n",
      "91008efedbdb903424a51e3cd18b8b80624dd4f4 is a new user.\n",
      "\n",
      "c0dc381d5ddf02f5182179c164ca65db6c8f572b is a new user.\n",
      "\n",
      "4cbca37009400bb5676ba54c2a4cc24ff0531cb7 is a new user.\n",
      "\n",
      "8084aef08dffb1c0323bc6af17f80b3cd9e2e7f3 is a new user.\n",
      "\n",
      "3a11e6d1f8aa60342e2486922edf199deadb2026 is a new user.\n",
      "\n",
      "dfd62f3ee786c4579b262421b33b8110d931733f is a new user.\n",
      "\n",
      "fa5d9eddc010bc3fc71f8a42db15e5dd4f1c18a3 is a new user.\n",
      "\n",
      "174b880b7340e2921b281493cfc6337bb7d99579 is a new user.\n",
      "\n",
      "6c99ae7e53e91f7e839444b2f00c8503a4a10fcf is a new user.\n",
      "\n",
      "d0a2c5ac5ce1bc3573224d910fe3adfb85d4ee3f is a new user.\n",
      "\n",
      "eb69f5a5465388b63fe66a410f7c58e17fb7bade is a new user.\n",
      "\n",
      "c802248ec0f960549720bb9f409fec7264b8b2f9 is a new user.\n",
      "\n",
      "954469357b2434a20c76e940eca93185141b7f9b is a new user.\n",
      "\n",
      "6270f977ebf8da9f90e2ff2d23bc570fa012b5ee is a new user.\n",
      "\n",
      "9bf0c40652e96b5178b552d2b296ec69b75a0b26 is a new user.\n",
      "\n",
      "c5d0030d32982330235e80d3395e412c38c552d1 is a new user.\n",
      "\n",
      "3fa44653315697f42410a30cb766a4eb102080bb is a new user.\n",
      "\n",
      "40e46bab2caa78cfe7150218f53ab7aed69a7b1e is a new user.\n",
      "\n",
      "91fc617d2632f554f364015e6e7fb10f3eacce1f is a new user.\n",
      "\n",
      "0fb08af6e4cd85e39f0c939097792a95be3431ee is a new user.\n",
      "\n",
      "339fbf843dcf3f7ec3a7f43c37b6a1ff37f5b817 is a new user.\n",
      "\n",
      "b82d692dde839cfe9fd6d309bb8c46d50089cc16 is a new user.\n",
      "\n",
      "2b8b819c92836d86d891d2c41bd25fdffd759898 is a new user.\n",
      "\n",
      "97e2a28f577accc9efa329a9f78f151994f917b4 is a new user.\n",
      "\n",
      "2744a71984cdf296fb94de2b9d5aa0f065ffb1ab is a new user.\n",
      "\n",
      "f986a1b01b2a75109baa39d637537b5124c111ab is a new user.\n",
      "\n",
      "2c04736f4d7b0696d2a063b1e61069af38660a94 is a new user.\n",
      "\n",
      "dc543ff951c23be4e9e8fdf6f39a23cef85f3b5e is a new user.\n",
      "\n",
      "d375c4189987e1029e61b35aff6d34d568e6705c is a new user.\n",
      "\n",
      "8ec7fd0c1acf1dbe44720e5eab44dbe524eb6caf is a new user.\n",
      "\n",
      "d71adb94c30db90d20b08bde62acb7ef1966083d is a new user.\n",
      "\n",
      "ec06a94669597cdc8d1360c6a906aa95333326e8 is a new user.\n",
      "\n",
      "2668af52749df405c6eee58d41c8afd434f617e2 is a new user.\n",
      "\n",
      "aabbd8b9388076451e70846a86cd4c8cb426873f is a new user.\n",
      "\n",
      "4d98756ff69be79de228c15432245766d4bf0316 is a new user.\n",
      "\n",
      "00fa0c8162aa95341f4da9defede8aae0675d3cc is a new user.\n",
      "\n",
      "fbd1b7d1bf19158773820cb45639362347979926 is a new user.\n",
      "\n",
      "3877e44d4632df125ec8a838aae59eb04d36f9d6 is a new user.\n",
      "\n",
      "d7d2d888ae04d16e994d6964214a1de81392ee04 is a new user.\n",
      "\n",
      "53456e59daa8f3a8498880dbebf595ae02be9de9 is a new user.\n",
      "\n",
      "146ad3a976414133d661a070b20f62edfc958d52 is a new user.\n",
      "\n",
      "168afb3dd9c92b4724171543c1c123aa26dd4cd1 is a new user.\n",
      "\n",
      "f629b337d01f254e0d685ccc66ec2fd5ddba78d9 is a new user.\n",
      "\n",
      "6e15cb4dcfdda600d893d230fb7aba1d9d0ff218 is a new user.\n",
      "\n",
      "60a90b3801bb9599b85a8448c6d44589fd1f984d is a new user.\n",
      "\n",
      "6a49ecaa3727df93615074768fe683491d274377 is a new user.\n",
      "\n",
      "33a1286454a3cff06e3c2324be746d2e23d7c270 is a new user.\n",
      "\n",
      "f02ef1e670319151d2d27f4215fb16a973f7de2f is a new user.\n",
      "\n",
      "7e543508a213f4f22e0cb54ecf2df9c370070a28 is a new user.\n",
      "\n",
      "6c6289326f70321f2b3e072daa44819efc55639a is a new user.\n",
      "\n",
      "53cc3e95468819addbfcaa1256b460984c581be3 is a new user.\n",
      "\n",
      "e6e0f68e948d7bcbf2ed9c4506a40a139a5e7bc7 is a new user.\n",
      "\n",
      "80ae8f21a060a7fdb7b5d4bb3d5f3021e624e19c is a new user.\n",
      "\n",
      "3288389bf9ef956a23a0a4ea86f60bf24ba7f69e is a new user.\n",
      "\n",
      "57f7c9671c77f73db905344afc45627cfe5cba77 is a new user.\n",
      "\n",
      "104bcda48463a99997f668b897c32234793cd514 is a new user.\n",
      "\n",
      "1db9458849024e54f89a807790230aa5701f112d is a new user.\n",
      "\n",
      "73a9b86c602d2ab5eb151b77f58bb6e7315dd7b2 is a new user.\n",
      "\n",
      "35b1d8452b62242a2ded2dc5ea05a0a407023cf7 is a new user.\n",
      "\n",
      "d964fc033291078031d117ed10adfb615948256d is a new user.\n",
      "\n",
      "38ae280090905f0778dd40727f89de3380fb2625 is a new user.\n",
      "\n",
      "1854daf178674bbac9a8ed3d481f95b76676b414 is a new user.\n",
      "\n",
      "491d048e26c51fcda0744355bf191d4ccf36f118 is a new user.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#统计总的用户\n",
    "unique_users_test = df_triplet_test['user'].unique()\n",
    "\n",
    "#为每个用户推荐的item的数目\n",
    "n_rec_items = 10\n",
    "\n",
    "#性能评价参数初始化，用户计算Percison和Recall\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合（对不同用户），用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "#残差平方和，用与计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "#对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    #测试集中该用户打过分的电影（用于计算评价指标的真实值）\n",
    "    if user not in users_index:   #user在训练集中没有出现过，新用户不能用协同过滤\n",
    "        print(str(user) + ' is a new user.\\n')\n",
    "        continue\n",
    "   \n",
    "    user_records_test= df_triplet_test[df_triplet_test.user == user]\n",
    "    \n",
    "    #对每个测试用户，计算该用户对训练集中未出现过的商品的打分，并基于该打分进行推荐（top n_rec_items）\n",
    "    #返回结果为DataFrame\n",
    "    rec_items = svd_CF_recommend(user)\n",
    "    \n",
    "    if rec_items.empty:\n",
    "        continue\n",
    "    \n",
    "    print(\"rec_items:\",rec_items)\n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['song']\n",
    "        \n",
    "        if item in user_records_test['song'].values:\n",
    "            n_hits += 1\n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    #计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['song']\n",
    "        score = user_records_test.iloc[i]['play_count']\n",
    "        \n",
    "        df1 = rec_items[rec_items.item_id == item]\n",
    "        if(df1.shape[0] == 0): #item不在推荐列表中，可能是新item在训练集中没有出现过，或者该用户已经打过分新item不能被协同过滤推荐\n",
    "            print(str(item) + ' is a new item or  user ' + str(user) +' already rated it.\\n')\n",
    "            continue\n",
    "        pred_score = df1['score'].values[0]\n",
    "        rss_test += (pred_score - score)**2     #残差平方和\n",
    "    \n",
    "    #推荐的item总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    \n",
    "    #真实item的总数\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "#Precision & Recall\n",
    "if n_total_rec_items != 0:\n",
    "    precision = n_hits / (1.0*n_total_rec_items)\n",
    "    recall = n_hits / (1.0*n_test_items)\n",
    "else:\n",
    "    precision = 0\n",
    "    recall = 0\n",
    "\n",
    "#覆盖度：推荐商品占总需要推荐商品的比例\n",
    "coverage = len(all_rec_items) / (1.0* n_items)\n",
    "\n",
    "#打分的均方误差\n",
    "rmse=np.sqrt(rss_test / df_triplet_test.shape[0])  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
